# coding:utf-8
# author:Ericam_
import re
import sys
from bs4 import BeautifulSoup
import urllib.request
import time

headers = ('User-Agent',
           'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1')
opener = urllib.request.build_opener()
opener.addheaders = {headers}
urllib.request.install_opener(opener)

def get_download(url):
    file = urllib.request.urlopen(url)  # 打开网页
    data = BeautifulSoup(file, from_encoding="utf8")    # 转换格式
    section_name = data.title.string    #title
    section_text = data.select('#bgdiv .border_l_r #content p')[0].text     #内容
    section_text = re.sub('\s+', '\r\n\t', section_text).strip('\r\n')   #内容
    fp = open('2.txt', 'a')     #打开文件
    fp.write(section_name + "\n")     # 写入title
    fp.write(section_text + "\n")#写入内容
    fp.close()#关闭文件
    pt_nexturl = 'var next_page = "(.*?)"'
    nexturl_num = re.compile(pt_nexturl).findall(str(data))
    nexturl_num = nexturl_num[0]
    return nexturl_num


if __name__ == '__main__':
    url = "https://www.biqudu.com/0_633/1626368.html"  # http://www.lewendu8.com/books/21/21335/6381842.html
    num = 228
    index = 1
    get_download(url)
    while (True):
        nexturl = get_download(url)
        index += 1
        sys.stdout.write("已下载:%.3f%%" % float(index / num * 100) + '\n')
        sys.stdout.flush()
        url = "https://www.biqudu.com/0_633/" + nexturl  # http://www.lewendu8.com/books/21/21335/
        if (nexturl == 'https://www.biqudu.com/0_633/'):  # http://www.lewendu8.com/books/21/21335/
            break
    print(time.clock())
